{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "PyTorch: Defining new autograd functions\n",
    "----------------------------------------\n",
    "\n",
    "A fully-connected ReLU network with one hidden layer and no biases, trained to\n",
    "predict y from x by minimizing squared Euclidean distance.\n",
    "\n",
    "This implementation computes the forward pass using operations on PyTorch\n",
    "Variables, and uses PyTorch autograd to compute gradients.\n",
    "\n",
    "In this implementation we implement our own custom autograd function to perform\n",
    "the ReLU function.\n",
    "\n",
    "Source Link: http://pytorch.org/tutorials/beginner/examples_autograd/two_layer_net_custom_function.html\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 style=\"background-image: linear-gradient( 135deg, #ABDCFF 10%, #0396FF 100%);\"> Orinal Tutorial code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 33014076.0\n",
      "1 30972098.0\n",
      "2 29985770.0\n",
      "3 26333562.0\n",
      "4 19902930.0\n",
      "5 12834515.0\n",
      "6 7556588.5\n",
      "7 4390074.5\n",
      "8 2710298.75\n",
      "9 1830699.625\n",
      "10 1347037.0\n",
      "11 1055569.625\n",
      "12 861751.0625\n",
      "13 721523.4375\n",
      "14 613950.875\n",
      "15 527829.1875\n",
      "16 457215.1875\n",
      "17 398364.21875\n",
      "18 348696.21875\n",
      "19 306496.15625\n",
      "20 270402.875\n",
      "21 239380.890625\n",
      "22 212616.8125\n",
      "23 189370.125\n",
      "24 169119.125\n",
      "25 151422.421875\n",
      "26 135919.578125\n",
      "27 122262.109375\n",
      "28 110216.265625\n",
      "29 99577.8046875\n",
      "30 90141.953125\n",
      "31 81741.2109375\n",
      "32 74248.6484375\n",
      "33 67554.71875\n",
      "34 61567.79296875\n",
      "35 56200.31640625\n",
      "36 51373.94921875\n",
      "37 47025.05859375\n",
      "38 43100.0234375\n",
      "39 39546.1875\n",
      "40 36327.53125\n",
      "41 33409.03515625\n",
      "42 30757.60546875\n",
      "43 28346.85546875\n",
      "44 26151.0390625\n",
      "45 24147.720703125\n",
      "46 22317.681640625\n",
      "47 20642.767578125\n",
      "48 19108.57421875\n",
      "49 17702.66015625\n",
      "50 16413.119140625\n",
      "51 15229.0400390625\n",
      "52 14140.33984375\n",
      "53 13138.51171875\n",
      "54 12215.564453125\n",
      "55 11364.8115234375\n",
      "56 10579.89453125\n",
      "57 9854.7900390625\n",
      "58 9184.501953125\n",
      "59 8564.5986328125\n",
      "60 7990.62646484375\n",
      "61 7459.1044921875\n",
      "62 6966.36962890625\n",
      "63 6509.31787109375\n",
      "64 6085.19775390625\n",
      "65 5691.236328125\n",
      "66 5325.08056640625\n",
      "67 4984.63330078125\n",
      "68 4667.98193359375\n",
      "69 4373.166015625\n",
      "70 4098.54345703125\n",
      "71 3842.66357421875\n",
      "72 3604.04931640625\n",
      "73 3381.523193359375\n",
      "74 3173.9140625\n",
      "75 2980.1005859375\n",
      "76 2799.1201171875\n",
      "77 2629.9072265625\n",
      "78 2471.822021484375\n",
      "79 2323.9111328125\n",
      "80 2185.55712890625\n",
      "81 2056.06494140625\n",
      "82 1934.7908935546875\n",
      "83 1821.2000732421875\n",
      "84 1714.7464599609375\n",
      "85 1614.9267578125\n",
      "86 1521.329833984375\n",
      "87 1433.564697265625\n",
      "88 1351.1539306640625\n",
      "89 1273.8135986328125\n",
      "90 1201.174072265625\n",
      "91 1132.9591064453125\n",
      "92 1068.8416748046875\n",
      "93 1008.6240844726562\n",
      "94 952.025146484375\n",
      "95 898.7752685546875\n",
      "96 848.7022705078125\n",
      "97 801.56787109375\n",
      "98 757.2022705078125\n",
      "99 715.450927734375\n",
      "100 676.1235961914062\n",
      "101 639.076416015625\n",
      "102 604.179931640625\n",
      "103 571.2871704101562\n",
      "104 540.2874145507812\n",
      "105 511.0595703125\n",
      "106 483.4914855957031\n",
      "107 457.4989318847656\n",
      "108 432.9806213378906\n",
      "109 409.8318786621094\n",
      "110 387.97686767578125\n",
      "111 367.3484802246094\n",
      "112 347.8816833496094\n",
      "113 329.4916687011719\n",
      "114 312.11676025390625\n",
      "115 295.7020263671875\n",
      "116 280.1944580078125\n",
      "117 265.5356750488281\n",
      "118 251.6796417236328\n",
      "119 238.58004760742188\n",
      "120 226.19732666015625\n",
      "121 214.48211669921875\n",
      "122 203.39981079101562\n",
      "123 192.91763305664062\n",
      "124 182.9999542236328\n",
      "125 173.6133270263672\n",
      "126 164.72801208496094\n",
      "127 156.3169403076172\n",
      "128 148.3542022705078\n",
      "129 140.81005859375\n",
      "130 133.66708374023438\n",
      "131 126.90184020996094\n",
      "132 120.4947738647461\n",
      "133 114.42040252685547\n",
      "134 108.66616821289062\n",
      "135 103.21378326416016\n",
      "136 98.04509735107422\n",
      "137 93.14307403564453\n",
      "138 88.49519348144531\n",
      "139 84.09058380126953\n",
      "140 79.9111099243164\n",
      "141 75.94760131835938\n",
      "142 72.18663787841797\n",
      "143 68.62029266357422\n",
      "144 65.23479461669922\n",
      "145 62.02239990234375\n",
      "146 58.974674224853516\n",
      "147 56.08095169067383\n",
      "148 53.33395767211914\n",
      "149 50.72690200805664\n",
      "150 48.251678466796875\n",
      "151 45.89992904663086\n",
      "152 43.66753005981445\n",
      "153 41.54750442504883\n",
      "154 39.533660888671875\n",
      "155 37.619720458984375\n",
      "156 35.8026123046875\n",
      "157 34.07432556152344\n",
      "158 32.433067321777344\n",
      "159 30.872838973999023\n",
      "160 29.390348434448242\n",
      "161 27.98135757446289\n",
      "162 26.640506744384766\n",
      "163 25.367591857910156\n",
      "164 24.156831741333008\n",
      "165 23.00533676147461\n",
      "166 21.910198211669922\n",
      "167 20.868465423583984\n",
      "168 19.877742767333984\n",
      "169 18.935218811035156\n",
      "170 18.03937339782715\n",
      "171 17.186267852783203\n",
      "172 16.3748836517334\n",
      "173 15.602479934692383\n",
      "174 14.868000030517578\n",
      "175 14.168902397155762\n",
      "176 13.50307559967041\n",
      "177 12.870216369628906\n",
      "178 12.267020225524902\n",
      "179 11.692877769470215\n",
      "180 11.146533012390137\n",
      "181 10.626420974731445\n",
      "182 10.130584716796875\n",
      "183 9.6593599319458\n",
      "184 9.209611892700195\n",
      "185 8.781604766845703\n",
      "186 8.374323844909668\n",
      "187 7.986157417297363\n",
      "188 7.616364479064941\n",
      "189 7.264134883880615\n",
      "190 6.928653717041016\n",
      "191 6.608942031860352\n",
      "192 6.304247856140137\n",
      "193 6.013945579528809\n",
      "194 5.737204074859619\n",
      "195 5.473515510559082\n",
      "196 5.222555160522461\n",
      "197 4.982944965362549\n",
      "198 4.7546467781066895\n",
      "199 4.537055492401123\n",
      "200 4.3294758796691895\n",
      "201 4.131722927093506\n",
      "202 3.9433321952819824\n",
      "203 3.7634785175323486\n",
      "204 3.592127799987793\n",
      "205 3.428689956665039\n",
      "206 3.272758960723877\n",
      "207 3.1243462562561035\n",
      "208 2.98241925239563\n",
      "209 2.8471460342407227\n",
      "210 2.7182013988494873\n",
      "211 2.59527850151062\n",
      "212 2.477956533432007\n",
      "213 2.366072177886963\n",
      "214 2.259195566177368\n",
      "215 2.157505512237549\n",
      "216 2.0602922439575195\n",
      "217 1.9675472974777222\n",
      "218 1.879105806350708\n",
      "219 1.7945671081542969\n",
      "220 1.7140244245529175\n",
      "221 1.637067437171936\n",
      "222 1.5636228322982788\n",
      "223 1.4937069416046143\n",
      "224 1.426749348640442\n",
      "225 1.3631479740142822\n",
      "226 1.302140474319458\n",
      "227 1.2439223527908325\n",
      "228 1.188383936882019\n",
      "229 1.1354527473449707\n",
      "230 1.0848199129104614\n",
      "231 1.0365127325057983\n",
      "232 0.9905102252960205\n",
      "233 0.9464414715766907\n",
      "234 0.90431809425354\n",
      "235 0.8642363548278809\n",
      "236 0.825908899307251\n",
      "237 0.7892940640449524\n",
      "238 0.7543168663978577\n",
      "239 0.7209768295288086\n",
      "240 0.6890276670455933\n",
      "241 0.6585588455200195\n",
      "242 0.6295422911643982\n",
      "243 0.6016789078712463\n",
      "244 0.5751615762710571\n",
      "245 0.5497236251831055\n",
      "246 0.525540828704834\n",
      "247 0.5023786425590515\n",
      "248 0.4802993834018707\n",
      "249 0.4591750204563141\n",
      "250 0.4389992356300354\n",
      "251 0.4196995794773102\n",
      "252 0.40127697587013245\n",
      "253 0.38366445899009705\n",
      "254 0.36693865060806274\n",
      "255 0.3508182168006897\n",
      "256 0.3354927599430084\n",
      "257 0.3207482397556305\n",
      "258 0.3067113757133484\n",
      "259 0.2932857871055603\n",
      "260 0.28046032786369324\n",
      "261 0.26826027035713196\n",
      "262 0.25654327869415283\n",
      "263 0.24534732103347778\n",
      "264 0.23465463519096375\n",
      "265 0.22444693744182587\n",
      "266 0.21470466256141663\n",
      "267 0.20531140267848969\n",
      "268 0.19638259708881378\n",
      "269 0.18782863020896912\n",
      "270 0.17969156801700592\n",
      "271 0.17192097008228302\n",
      "272 0.16442108154296875\n",
      "273 0.1572970598936081\n",
      "274 0.15047690272331238\n",
      "275 0.14392535388469696\n",
      "276 0.1376980096101761\n",
      "277 0.1317261904478073\n",
      "278 0.1260254979133606\n",
      "279 0.12056288868188858\n",
      "280 0.11536846309900284\n",
      "281 0.11037227511405945\n",
      "282 0.10562607645988464\n",
      "283 0.10107224434614182\n",
      "284 0.09668958932161331\n",
      "285 0.09256551414728165\n",
      "286 0.08854427933692932\n",
      "287 0.08476962149143219\n",
      "288 0.08111885190010071\n",
      "289 0.0776192843914032\n",
      "290 0.07427306473255157\n",
      "291 0.071100153028965\n",
      "292 0.06805527955293655\n",
      "293 0.06511513143777847\n",
      "294 0.0623306930065155\n",
      "295 0.05965312197804451\n",
      "296 0.05710066854953766\n",
      "297 0.054628558456897736\n",
      "298 0.052287548780441284\n",
      "299 0.050060149282217026\n",
      "300 0.04791659861803055\n",
      "301 0.04585902765393257\n",
      "302 0.04390404373407364\n",
      "303 0.04202497377991676\n",
      "304 0.040233779698610306\n",
      "305 0.03851644694805145\n",
      "306 0.0368657223880291\n",
      "307 0.035304006189107895\n",
      "308 0.03378855437040329\n",
      "309 0.032353248447179794\n",
      "310 0.030977562069892883\n",
      "311 0.029651403427124023\n",
      "312 0.028405142948031425\n",
      "313 0.02720700204372406\n",
      "314 0.0260466281324625\n",
      "315 0.024954738095402718\n",
      "316 0.02390369214117527\n",
      "317 0.022898752242326736\n",
      "318 0.021929671987891197\n",
      "319 0.02100413851439953\n",
      "320 0.020117105916142464\n",
      "321 0.019266780465841293\n",
      "322 0.018459845334291458\n",
      "323 0.017675908282399178\n",
      "324 0.016942350193858147\n",
      "325 0.01623787172138691\n",
      "326 0.015555662102997303\n",
      "327 0.014906282536685467\n",
      "328 0.014280903153121471\n",
      "329 0.01368409302085638\n",
      "330 0.013115848414599895\n",
      "331 0.012571672908961773\n",
      "332 0.012050933204591274\n",
      "333 0.01155149657279253\n",
      "334 0.011071830056607723\n",
      "335 0.010618075728416443\n",
      "336 0.010179744102060795\n",
      "337 0.00976420659571886\n",
      "338 0.00936319399625063\n",
      "339 0.00897684320807457\n",
      "340 0.008615781553089619\n",
      "341 0.008259537629783154\n",
      "342 0.007928203791379929\n",
      "343 0.0076066614128649235\n",
      "344 0.007301232311874628\n",
      "345 0.007006747182458639\n",
      "346 0.006716860458254814\n",
      "347 0.006448232103139162\n",
      "348 0.006190649699419737\n",
      "349 0.005941507872194052\n",
      "350 0.005709324963390827\n",
      "351 0.005481124855577946\n",
      "352 0.005266945343464613\n",
      "353 0.005056160502135754\n",
      "354 0.004854677710682154\n",
      "355 0.004667287226766348\n",
      "356 0.00448642997071147\n",
      "357 0.004310122225433588\n",
      "358 0.0041443658992648125\n",
      "359 0.003984262701123953\n",
      "360 0.003832496702671051\n",
      "361 0.0036846441216766834\n",
      "362 0.0035448686685413122\n",
      "363 0.003411193611100316\n",
      "364 0.0032792326528578997\n",
      "365 0.003155139507725835\n",
      "366 0.0030368140432983637\n",
      "367 0.0029195104725658894\n",
      "368 0.002811953192576766\n",
      "369 0.0027107116766273975\n",
      "370 0.00261088740080595\n",
      "371 0.0025156212504953146\n",
      "372 0.0024209979455918074\n",
      "373 0.002334759570658207\n",
      "374 0.0022522276267409325\n",
      "375 0.002173313871026039\n",
      "376 0.002095114439725876\n",
      "377 0.00202330038882792\n",
      "378 0.0019503177609294653\n",
      "379 0.0018848481122404337\n",
      "380 0.0018203867366537452\n",
      "381 0.001755950041115284\n",
      "382 0.00169463106431067\n",
      "383 0.0016384038608521223\n",
      "384 0.0015812413766980171\n",
      "385 0.0015287547139450908\n",
      "386 0.0014788854168727994\n",
      "387 0.0014282757183536887\n",
      "388 0.0013813900295644999\n",
      "389 0.0013365451013669372\n",
      "390 0.0012930340599268675\n",
      "391 0.0012500927550718188\n",
      "392 0.0012125420616939664\n",
      "393 0.0011741864727810025\n",
      "394 0.001136835664510727\n",
      "395 0.0011009291047230363\n",
      "396 0.0010651829652488232\n",
      "397 0.0010332706151530147\n",
      "398 0.0010006135562434793\n",
      "399 0.000969582877587527\n",
      "400 0.0009401569259352982\n",
      "401 0.0009127026423811913\n",
      "402 0.0008860370726324618\n",
      "403 0.0008602559682913125\n",
      "404 0.0008344394736923277\n",
      "405 0.0008109149057418108\n",
      "406 0.0007868682150729001\n",
      "407 0.0007643437129445374\n",
      "408 0.0007421437767334282\n",
      "409 0.0007203964050859213\n",
      "410 0.0007007990498095751\n",
      "411 0.0006817550165578723\n",
      "412 0.0006623465451411903\n",
      "413 0.0006438774871639907\n",
      "414 0.0006263290415517986\n",
      "415 0.0006087308865971863\n",
      "416 0.0005914103821851313\n",
      "417 0.0005747953546233475\n",
      "418 0.0005608684150502086\n",
      "419 0.0005450196331366897\n",
      "420 0.0005302880890667439\n",
      "421 0.000516871630679816\n",
      "422 0.0005034647765569389\n",
      "423 0.0004909792332910001\n",
      "424 0.0004785972705576569\n",
      "425 0.00046541192568838596\n",
      "426 0.0004530898586381227\n",
      "427 0.0004422176571097225\n",
      "428 0.00043101096525788307\n",
      "429 0.00042021804256364703\n",
      "430 0.0004099572543054819\n",
      "431 0.00040016911225393414\n",
      "432 0.00039151142118498683\n",
      "433 0.00038184894947335124\n",
      "434 0.0003719590895343572\n",
      "435 0.00036389229353517294\n",
      "436 0.00035553579800762236\n",
      "437 0.00034661131212487817\n",
      "438 0.0003380749258212745\n",
      "439 0.0003315811336506158\n",
      "440 0.00032264547189697623\n",
      "441 0.0003160503983963281\n",
      "442 0.0003086808428633958\n",
      "443 0.0003021802403964102\n",
      "444 0.0002954314695671201\n",
      "445 0.00028872487018816173\n",
      "446 0.00028209216543473303\n",
      "447 0.00027571996906772256\n",
      "448 0.0002697384334169328\n",
      "449 0.00026386076933704317\n",
      "450 0.0002574942191131413\n",
      "451 0.0002528493059799075\n",
      "452 0.0002474599168635905\n",
      "453 0.0002427824801998213\n",
      "454 0.00023705170315224677\n",
      "455 0.00023261108435690403\n",
      "456 0.00022743493900634348\n",
      "457 0.0002239200402982533\n",
      "458 0.00021898426348343492\n",
      "459 0.00021420774282887578\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "460 0.00021031788492109627\n",
      "461 0.0002061791019514203\n",
      "462 0.00020195210527163\n",
      "463 0.00019812598475255072\n",
      "464 0.00019394831906538457\n",
      "465 0.00019033234275411814\n",
      "466 0.00018701166845858097\n",
      "467 0.0001830069231800735\n",
      "468 0.0001798015582608059\n",
      "469 0.00017638066492509097\n",
      "470 0.00017323627253063023\n",
      "471 0.00016992753080558032\n",
      "472 0.00016698843683116138\n",
      "473 0.00016396869614254683\n",
      "474 0.0001605374418431893\n",
      "475 0.00015738226647954434\n",
      "476 0.0001546780113130808\n",
      "477 0.00015191755665000528\n",
      "478 0.00014942808775231242\n",
      "479 0.00014680263120681047\n",
      "480 0.00014404243847820908\n",
      "481 0.0001417940075043589\n",
      "482 0.00013912047143094242\n",
      "483 0.00013693823711946607\n",
      "484 0.00013444662909023464\n",
      "485 0.00013254550867713988\n",
      "486 0.0001297335111303255\n",
      "487 0.00012766171130351722\n",
      "488 0.00012503366451710463\n",
      "489 0.00012348050950095057\n",
      "490 0.00012129628157708794\n",
      "491 0.00011923334386665374\n",
      "492 0.00011711245315382257\n",
      "493 0.00011535739758983254\n",
      "494 0.00011370572610758245\n",
      "495 0.00011150680074933916\n",
      "496 0.00010982639651047066\n",
      "497 0.00010767151252366602\n",
      "498 0.0001055961474776268\n",
      "499 0.00010440174082759768\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable\n",
    "\n",
    "\n",
    "class MyReLU(torch.autograd.Function):\n",
    "    \"\"\"\n",
    "    We can implement our own custom autograd Functions by subclassing\n",
    "    torch.autograd.Function and implementing the forward and backward passes\n",
    "    which operate on Tensors.\n",
    "    \"\"\"\n",
    "\n",
    "    @staticmethod\n",
    "    def forward(ctx, input):\n",
    "        \"\"\"\n",
    "        In the forward pass we receive a Tensor containing the input and return\n",
    "        a Tensor containing the output. ctx is a context object that can be used\n",
    "        to stash information for backward computation. You can cache arbitrary\n",
    "        objects for use in the backward pass using the ctx.save_for_backward method.\n",
    "        \"\"\"\n",
    "        ctx.save_for_backward(input)\n",
    "        return input.clamp(min=0)\n",
    "\n",
    "    @staticmethod\n",
    "    def backward(ctx, grad_output):\n",
    "        \"\"\"\n",
    "        In the backward pass we receive a Tensor containing the gradient of the loss\n",
    "        with respect to the output, and we need to compute the gradient of the loss\n",
    "        with respect to the input.\n",
    "        \"\"\"\n",
    "        input, = ctx.saved_tensors\n",
    "        grad_input = grad_output.clone()\n",
    "        grad_input[input < 0] = 0\n",
    "        return grad_input\n",
    "\n",
    "\n",
    "dtype = torch.FloatTensor\n",
    "# dtype = torch.cuda.FloatTensor # Uncomment this to run on GPU\n",
    "\n",
    "# N is batch size; D_in is input dimension;\n",
    "# H is hidden dimension; D_out is output dimension.\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# Create random Tensors to hold input and outputs, and wrap them in Variables.\n",
    "x = Variable(torch.randn(N, D_in).type(dtype), requires_grad=False)\n",
    "y = Variable(torch.randn(N, D_out).type(dtype), requires_grad=False)\n",
    "\n",
    "# Create random Tensors for weights, and wrap them in Variables.\n",
    "w1 = Variable(torch.randn(D_in, H).type(dtype), requires_grad=True)\n",
    "w2 = Variable(torch.randn(H, D_out).type(dtype), requires_grad=True)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "for t in range(500):\n",
    "    # To apply our Function, we use Function.apply method. We alias this as 'relu'.\n",
    "    relu = MyReLU.apply\n",
    "\n",
    "    # Forward pass: compute predicted y using operations on Variables; we compute\n",
    "    # ReLU using our custom autograd operation.\n",
    "    y_pred = relu(x.mm(w1)).mm(w2)\n",
    "\n",
    "    # Compute and print loss\n",
    "    loss = (y_pred - y).pow(2).sum()\n",
    "    print(t, loss.data[0])\n",
    "\n",
    "    # Use autograd to compute the backward pass.\n",
    "    loss.backward()\n",
    "\n",
    "    # Update weights using gradient descent\n",
    "    w1.data -= learning_rate * w1.grad.data\n",
    "    w2.data -= learning_rate * w2.grad.data\n",
    "\n",
    "    # Manually zero the gradients after updating weights\n",
    "    w1.grad.data.zero_()\n",
    "    w2.grad.data.zero_()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
